import requests
import json
from pyquery import PyQuery as pq

URL = 'https://gitlab.xthktech.cn/?non_archived=true&page={}&sort=latest_activity_desc'


headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
    'cookie': 'OUTFOX_SEARCH_USER_ID_NCOO=596770881.5366416; experimentation_subject_id=IjQ0ZGUyOTk0LWEzYTgtNDJiNy1iMTU3LWVlZWNiYmYzZTQ4MiI%3D--5efc25a97d4a350933456fbcd142d1a9ce81024d; sidebar_collapsed=false; auto_devops_settings_dismissed=true; event_filter=all; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22601343%22%2C%22first_id%22%3A%221724ee5c190b99-02ff06e12fea13-7711a3e-2073600-1724ee5c191944%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%221724ee5c190b99-02ff06e12fea13-7711a3e-2073600-1724ee5c191944%22%7D; diff_view=parallel; collapsed_gutter=true; _gitlab_session=e5bb303741515f1330d56322f9f5f1ce; token=%2209de8eeb5d1ae1973b4f9b5877a9b048%22'
}

GIT_URL = 'git@gitlab.xthktech.cn:{}.git'
projects = []
for i in range(1, 6):
    url = URL.format(i)
    r = requests.get(url, headers=headers)

    d = pq(r.text)

    for a in d('.project-row'):
        da = pq(a)
        href = da('h2.prepend-top-8 a').attr.href
        description = da('.description p').text()
        git_url = GIT_URL.format(href[1:])
        projects.append({
            'url': git_url,
            'description': description
        })

j = json.dumps(projects)
print(j)